# -*- coding:utf-8 -*-
# Created by Jin(jinzhencheng@outlook.com) at 2018/8/3 17:01

from scrapy import Spider
from lxml import etree

from bd_spider.config import SpiderConfig
from bd_spider.dal import baijiahao_dal
from bd_spider.config import GenerationConfig
from bd_spider.action import simulation
from bd_spider.items import Article
import time


class ArticleSpider(Spider):

    name = SpiderConfig.ARTICLE_SPIDER_NAME

    # start_requests 必须返回Request 对象，故将selenium处理过程在parse中完成
    start_urls = ["http://www.baidu.com"]

    def parse(self, response):
        result =  baijiahao_dal.list()
        for index, item in enumerate(result):
            url = SpiderConfig.author_home_url(item.app_id)
            simulation.request(url)
            time.sleep(GenerationConfig.DEFAULT_WAIT_TIME)
            simulation.browser.execute_script(SpiderConfig.GO_PAGE_BOTTOM_JS)
            time.sleep(GenerationConfig.DEFAULT_WAIT_TIME)
            final_url, source = simulation.response()
            article = Article()
            article["baijiahao_id"] = item.id
            page = etree.HTML(source)
            for sel in page.xpath("//*[@id='article']/section/ul/li"):
                article_id = sel.xpath("./@article-id")[0]
                article["article_id"] = article_id
                title = sel.xpath("./div/a/div[@class='text']/h2/text()")[0]
                article["title"] = title
                src = sel.xpath("./div/a/@href")[0]
                article["src"] = src
                view= sel.xpath("./div/a/div[@class='text']/div[@class='text-extra']/div[@class='pv']/text()")
                if view:
                    view = view[0][:len(view) - 3]
                    article["view"] = view
                comment= sel.xpath("./div/a/div[@class='text']/div[@class='text-extra']/div[@class='comment']/text()")
                if comment:
                    comment = comment[0][:len(comment) - 3]
                    article["comment"] = comment
                publish_time = sel.xpath("./div/a/div[@class='text']/div[@class='text-extra']/div[@class='time']/text()")
                if publish_time:
                    article["publish_time"] = publish_time[0]
                yield article
